package com.guiji.quartz.domain;

import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONNull;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.quartz.config.ProxyConfig;
import com.guiji.quartz.task.DataSaveUtil;
import com.guiji.quartz.util.DelPicUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description: 需要代理  TODO
 * @author: wangxw
 * @date 2021/11/29 18:28
 */
public class Sginvestors {
	private static final Logger log = LoggerFactory.getLogger(Sginvestors.class);

	public static void mainMethod() throws Exception {
		HttpResponse response = null;
		try {
			response = HttpRequest.get("https://sginvestors.io/news/publishers/latest/")
				.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
				.header("Accept-Encoding", "gzip, deflate")
				.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
				.setHttpProxy(ProxyConfig.url, ProxyConfig.ip)
				.setConnectionTimeout(30000)
				.execute();

			if (null != response && response.getStatus() == 200) {
				List<HttpCookie> cookies = response.getCookies();
				String resp = response.body();
				Set<String> resUrl = getUrl(resp);
				getContent(resUrl, cookies);
			} else {
				log.error("https://sginvestors.io/news/publishers/latest/ " + String.valueOf(response.getStatus()));
			}

		} catch (Exception e) {
			log.error("访问 https://sginvestors.io/news/publishers/latest/ 失败" + e);
			return;
		}


	}

	private static void getContent(Set<String> listContent, List<HttpCookie> cookies) throws MalformedURLException, URISyntaxException, InterruptedException {
		for (String url : listContent) {
			Thread.sleep(1000);
			if (!url.startsWith("https")) {
				url = new StringBuffer("https://sginvestors.io").append(url).toString();
			}
			if (null != cookies && cookies.size() > 0) {
				CookieManager ccm = new CookieManager();
				CookieStore CookieStore = ccm.getCookieStore();
				for (HttpCookie cookie : cookies) {
					URL domain = new URL("https://sginvestors.io");
					CookieStore.add(domain.toURI(), cookie);
				}
				HttpRequest.setCookieManager(ccm);
			}
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			//这个网站首页外链了好几个网站的内容链接
			if (url.startsWith("https://www.todayonline.com")) {
				getContentOfTodayonline(url, cookies);
			} else if (url.startsWith("https://www.channelnewsasia.com/")) {
				String doc;
				JXDocument jxDocument;
				try {
					HttpResponse response = HttpRequest.get(url)
						.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
						.header("Refer", "https://www.todayonline.com/singapore/tocs-lawyer-lim-tean-says-shanmugam-wrong-use-pofma-over-post-alleging-police-bullying-agc")
						.setHttpProxy(ProxyConfig.url, ProxyConfig.ip)
						.setConnectionTimeout(30000).execute();
					doc = response.body();
					jxDocument = new JXDocument(doc);
				} catch (Exception e) {
					log.info("代理失败2" + e);
					continue;
				}
				String title = null;
				try {
					title = (String) jxDocument.sel("//h1/text()").get(0);
				} catch (Exception e) {
					log.error("标题错误-------------" + url + "-------------------标题错误");
					continue;
				}
				StringBuffer content = new StringBuffer();
				String contentStr = null;
				try {

					Document document = Jsoup.parse(doc);
					Element mainArea = document.getElementsByClass("text").get(0);
					Elements pElements = mainArea.select("P");
					if (null != pElements && pElements.size() > 0) {
						for (Element pdom : pElements) {
							String temp = pdom.text();
							if (null != temp && !temp.toString().contains("Your support through more subscriptions can help us") && !temp.toString().contains("Only the headline and picture of this report")) {
								content.append(temp + "$$$");
								pdom.remove();
							}
						}
					}
					contentStr = content.toString();
				} catch (Exception e) {
					continue;
				}

				List<Object> srcImgs = new ArrayList<>();
				List<String> newImgs = new ArrayList<>();
				String img = null;
				try {
					int imgSize = jxDocument.sel("//section[@id='block-mc-cna-theme-mainpagecontent']/article//img/@src").size();
					if (imgSize > 0) {
						String temp = null;
						for (int k = 0; k < imgSize; k++) {
							temp = jxDocument.sel("//section[@id='block-mc-cna-theme-mainpagecontent']/article//img/@src").get(k).toString();
							img = temp.strip();
							srcImgs.add(img);
							break;
						}
					}

				} catch (Exception e) {
					log.error("可能没有图片");
					img = "";
				} finally {
					if (srcImgs.size() > 0) {
						newImgs = DelPicUtil.delPic(srcImgs);
					}
				}

				Date date = new Date();
				if (contentStr.length() > 50000 || contentStr.replaceAll("\\$|[\\s\\p{Zs}]", "").length() < 20 || newImgs.size() > 20) {
					continue;
				}
				try {
					System.out.println("=======================================================");
					System.out.println(url);
					System.out.println(title);

					for (Object imgitem : srcImgs) {
						System.out.println(" " + imgitem);
					}
					System.out.println(contentStr);
					System.out.println("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
					DataSaveUtil.saveData(title, contentStr, newImgs, url, date, "https://sginvestors.io/news/publishers/latest/", "Sginvestors-EnglishBusiness");
					log.info(Thread.currentThread().getName() + "-----------------Sginvestors-EnglishBusiness----------------");
				} catch (Exception e) {
					continue;
				}
			}
		}
	}

	private static Set<String> getUrl(String content) {

		Set<String> res = new HashSet<>();
		String pattern = "href='https://www.*?'";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		String temp = null;
		while (m.find()) {
			temp = m.group();
			temp = temp.replace("href=", "");
			temp = temp.replace("'", "");
			if (temp.contains("https://www.theedgesingapore.com") || temp.contains("www.businesstimes.com")) {
				continue;
			}
			res.add(temp);
		}
		return res;
	}


	private static void getContentOfTodayonline(String url, List<HttpCookie> cookies) {

		HttpResponse response = null;
		String ajaxBaseUrl = "https://www.todayonline.com/api/v3/article/%s";
		String articleId = null;
		String imgUrl = null;
		String title = null;
		String content = null;
		List<Object> srcImgs = new ArrayList<>();
		List<String> newImgs = new ArrayList<>();
		try {
			response = HttpRequest.get(url)
				.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
				.header("Accept-Encoding", "gzip, deflate")
				.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
				.setHttpProxy(ProxyConfig.url, ProxyConfig.ip)
				.setConnectionTimeout(30000)
				.execute();
			if (response.getStatus() == 200) {
				String doc = response.body();
				String pattern = "var articleId = ";
				if (doc.contains(pattern)) {
					articleId = doc.substring(doc.indexOf(pattern) + 17, doc.indexOf(pattern) + 25);
				}

				String pattern1 = "=\"og:image\" content=\"(.*?)\"";
				Pattern p1 = Pattern.compile(pattern1);
				Matcher m1 = p1.matcher(response.body());
				if (m1.find()) {
					imgUrl = m1.group();
					imgUrl = imgUrl.replace("=\"og:image\" content=", "");
					imgUrl = imgUrl.replace("\"", "");
				}
				srcImgs.add(imgUrl);


			}

			if (articleId != null) {
				String ajaxurl = String.format(ajaxBaseUrl, articleId);
				response = HttpRequest.get(ajaxurl)
					.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
					.header("Accept-Encoding", "gzip, deflate")
					.header("Refer", "https://www.todayonline.com/singapore/tocs-lawyer-lim-tean-says-shanmugam-wrong-use-pofma-over-post-alleging-police-bullying-agc")
					.header("Host", "www.todayonline.com")
					.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
					.setHttpProxy(ProxyConfig.url, ProxyConfig.ip)
					.setConnectionTimeout(30000)
					.execute();
				if (response.getStatus() == 200) {
					JSONObject jobj = JSONUtil.parseObj(response.body());
					JSONObject node = (JSONObject) jobj.get("node");
					title = (String) node.get("title");
					String body = (String) node.get("body");

					title = unicodeToUtf8(title);
					content = body;
				}
			}

			if (srcImgs.size() > 0) {
				newImgs = DelPicUtil.delPic(srcImgs);
			}

			if (null != content) {
				content = content.replaceAll("\r\n", "\\$\\$\\$")
					.replaceAll("\r", "	")
					.replaceAll("\n", "")
					.replaceAll("\t", "")
					.replaceAll("<(.|\n)*?>", " ")
					.replaceAll("&nbsp;", " ");

				Date date = new Date();
				try {
					System.out.println("=======================================================");
					System.out.println(url);
					System.out.println(title);

					for (Object imgitem : srcImgs) {
						System.out.println(" " + imgitem);
					}
					System.out.println(content);
					System.out.println("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
					DataSaveUtil.saveData(title, content, newImgs, url, date, "https://sginvestors.io/news/publishers/latest/", "Sginvestors-EnglishBusiness");
					log.info(Thread.currentThread().getName() + "-----------------Sginvestors-EnglishBusiness----------------");
				} catch (Exception e) {
					log.error(e.toString());
					return;
				}
			}

		} catch (Exception e) {
			log.error("getContentOfTodayonline failed");
			return;
		}


	}


	public static String unicodeToUtf8(String theString) {
		char aChar;
		int len = theString.length();
		StringBuffer outBuffer = new StringBuffer(len);
		for (int x = 0; x < len; ) {
			aChar = theString.charAt(x++);
			if (aChar == '\\') {
				aChar = theString.charAt(x++);
				if (aChar == 'u') {
					// Read the xxxx
					int value = 0;
					for (int i = 0; i < 4; i++) {
						aChar = theString.charAt(x++);
						switch (aChar) {
							case '0':
							case '1':
							case '2':
							case '3':
							case '4':
							case '5':
							case '6':
							case '7':
							case '8':
							case '9':
								value = (value << 4) + aChar - '0';
								break;
							case 'a':
							case 'b':
							case 'c':
							case 'd':
							case 'e':
							case 'f':
								value = (value << 4) + 10 + aChar - 'a';
								break;
							case 'A':
							case 'B':
							case 'C':
							case 'D':
							case 'E':
							case 'F':
								value = (value << 4) + 10 + aChar - 'A';
								break;
							default:
								throw new IllegalArgumentException(
									"Malformed   \\uxxxx   encoding.");
						}
					}
					outBuffer.append((char) value);
				} else {
					if (aChar == 't')
						aChar = '\t';
					else if (aChar == 'r')
						aChar = '\r';
					else if (aChar == 'n')
						aChar = '\n';
					else if (aChar == 'f')
						aChar = '\f';
					outBuffer.append(aChar);
				}
			} else
				outBuffer.append(aChar);
		}
		return outBuffer.toString();
	}

//	public static void main(String[] args) throws Exception {
//		mainMethod();
//	}

}



